#define FEELINGCAUTIOUS
// License Notification
//======================================================
// Copyright (c) 2010 Channel Intelligence Inc.
// All rights reserved. This program and the accompanying materials
// are made available under the terms of the Eclipse Public License v1.0
// which accompanies this distribution, and is available at
// http://www.eclipse.org/legal/epl-v10.html
//
// ======================================================




using System;
using System.IO;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Reflection;
using System.Text.RegularExpressions;
using CIP4.Common;
using CIP4.Simple;
using CIP4.RulesEngine;
using MongoDB.Bson;
using MongoDB.Driver;
using MongoDB.Driver.GridFS;

namespace CIP4.DataProvider.MongoDB
{
    public class DataProvider : Simple.DataProvider, IDataProvider
    {
        private readonly MongoServer _mongo;
        private readonly MongoDatabase _mongoDB;

        IDataProvider Cache { get; set; }

        #region IDataProvider Members

        public string ConnectionString { get; private set; }

        public DataProvider (CIP4ConnectionAttribute dataProviderConnection, IDataProvider cachingProvider, LoggerDelegate logger)
            : base (dataProviderConnection)
        {
            Cache = cachingProvider;
            if (!(dataProviderConnection is CIP4MongoDBConnectionAttribute))
                LoggerWrapper(MajorEnum.Error, MinorEnum.UnexpectedConnectionType,
                    "{0} is not an acceptable connection type", new object[] { dataProviderConnection.ToString() }, logger);
            else
            {
                LoggerWrapper(MajorEnum.Trace, MinorEnum.InitializingProvider,
                    "Initalizing Provider: {0}", new object[] { dataProviderConnection }, logger);
                CIP4MongoDBConnectionAttribute connection = (CIP4MongoDBConnectionAttribute)dataProviderConnection;
                try
                {
                    ConnectionString = connection.ToString();
                    _mongo = MongoServer.Create(string.Format("mongodb://{0}:{1}", connection.Server, connection.Port));
                    if (connection.Timeout > 0)
                        _mongo.Settings.ConnectTimeout = new TimeSpan(0, 0, connection.Timeout/1000);
                    _mongoDB = _mongo[connection.Database];
                }
                catch (Exception ex)
                {
                    LoggerWrapper(MajorEnum.Error, MinorEnum.InitializingProvider, 
                        "Error initializing MongoDB: {0}", new object[] { ex }, logger);
                }
            }
        }

        public void DeleteCollection(CollectionType collection, LoggerDelegate logger)
        {
            if (!LoggerWrapper(MajorEnum.Trace, MinorEnum.ActionStart,
                "Started {0}", new object[] { MinorEnum.DeleteCollection }, logger))
                return;

            MongoCollection  mongoCollection = _mongoDB[collection.Name];
            mongoCollection.Remove(new QueryDocument(), RemoveFlags.None);

            MongoGridFS fs = new MongoGridFS(_mongoDB, new MongoGridFSSettings {Root = collection.Name + "_filedata"});
            fs.Delete(new QueryDocument());
            
            LoggerWrapper(MajorEnum.Information, MinorEnum.DeleteCollection,
                "Collection deleted {0}", new object[] { collection }, logger);
        }

        public override IEnumerable<object> GetItems(CollectionType collection, IBooleanValue selectionExpression, CursorType cursor)
        {
            List<object> items = new List<object>();
            if (!LoggerWrapper(MajorEnum.Trace, MinorEnum.ActionStart,
                "Started {0}", new object[] { MinorEnum.GetItems }, cursor.Logger))
                return items;

            try
            {
                ISerializerType serializer = cursor.GetSerializer(SerializerKindEnum.CIP4Json);

                string lookForExternalId = GetExternalIdValueFromSelectionExpression(selectionExpression, serializer);

                if (Cache != null && !string.IsNullOrEmpty(lookForExternalId))
                {
                    items.AddRange(Cache.GetItems(collection, selectionExpression, cursor));
                    if (items.Count > 0)
                    {
                        LoggerWrapper(MajorEnum.Trace, MinorEnum.CacheHit, "Cache hit: {0}", new[] { items[0] }, cursor.Logger);
                        return items;
                    }
                }


                if (!string.IsNullOrEmpty(lookForExternalId))
                {
                    try
                    {
                        MongoCollection  mongoCollection = _mongoDB[collection.Name];

                        QueryDocument qdoc = new QueryDocument(new BsonElement("_id", lookForExternalId));

                        BsonDocument mongoDoc = mongoCollection.FindOneAs<BsonDocument>(qdoc);
  
                        if (mongoDoc != null)
                        {
                            if (mongoDoc.First() == null)
                                LoggerWrapper(MajorEnum.Error, MinorEnum.GetItems, "Find returned an empty document from collection {0}, selection expression {1}", new object[] { collection.Name, selectionExpression }, cursor.Logger);
                            else if (mongoDoc.First().Name == "$err")
                                LoggerWrapper(MajorEnum.Error, MinorEnum.GetItems, "Find in collection {0}, selection expression {1} encountered an error", new object[] { collection.Name, selectionExpression, mongoDoc.ToString() }, cursor.Logger);
                            else
                            {
                                items.Add(serializer.GetObjectForData(mongoDoc));
                                GetItemsStreamResolver(serializer, collection, mongoDoc["_id"].ToString());
                                if (Cache != null)
                                    Cache.SaveItems(items, collection, cursor.GetSerializer);
                            }
                        }
                        return items;
                    }
                    catch (FileNotFoundException)
                    {
                        return items;
                    }
                }

                {
                    int skip;
                    int.TryParse(cursor.Marker, out skip);

                    LoggerResponseEnum whatToDo = LoggerResponseEnum.keepGoing;
                    QueryDocument jsonQuery = BuildQuery(
                        selectionExpression,
                        collection,
                        serializer,
                        delegate(MajorEnum status, MinorEnum minorStatus, string message, object[] paras)
                        {
                            cursor.Logger(status, minorStatus, message, paras);
                            if (status == MajorEnum.Error)
                                whatToDo = LoggerResponseEnum.giveUp;
                            return LoggerResponseEnum.keepGoing;
                        });

                    cursor.Logger(MajorEnum.Trace, MinorEnum.QueryText, "Mongo query:{0}",
                                  new object[] {jsonQuery.ToJson()});

                    if (whatToDo == LoggerResponseEnum.giveUp)
                        return items;

                    if (cursor.FailOnLinearSearch && IsLinearSearch(jsonQuery, serializer))
                        if (!LoggerWrapper(MajorEnum.Error, MinorEnum.FailOnLinearSearch,
                            "Failed on linear search accessing {0}", new object[] { collection }, cursor.Logger))
                            return items;

                    MongoCollection  mongoCollection = _mongoDB[collection.Name];

                    MongoCursor<BsonDocument> all = mongoCollection.FindAs<BsonDocument>(jsonQuery);

                    all.SetBatchSize(cursor.PageSize);

                    all.SetLimit(cursor.PageSize);

                    all.SetSkip((int)((cursor.PageNumber - 1) * cursor.PageSize));
                    
                    foreach (BsonDocument mongoDoc in all)
                    {
                        if (mongoDoc.First() == null)
                            LoggerWrapper(MajorEnum.Error, MinorEnum.GetItems, "Find returned an empty document from collection {0}, selection expression {1}", new object[] { collection.Name, selectionExpression }, cursor.Logger);
                        else if (mongoDoc.First().Name == "$err")
                            LoggerWrapper(MajorEnum.Error, MinorEnum.GetItems, "Find in collection {0}, selection expression {1} encountered an error", new object[] { collection.Name, selectionExpression, mongoDoc.ToString() }, cursor.Logger);
                        else
                        {
                            object item = serializer.GetObjectForData(mongoDoc);
                            if (CompareProperties(selectionExpression, item, serializer))

                                items.Add(serializer.GetObjectForData(mongoDoc));
                            GetItemsStreamResolver(serializer, collection, mongoDoc["_id"].ToString());
                        }
                    }
                    if (Cache != null)
                        Cache.SaveItems(items, collection, cursor.GetSerializer);
                    return items;
                }
            }
            //catch (Exception ex)  // would like to do this here, but not quite sure what to return...
            //{
            //    LoggerWrapper(MajorEnum.Error, MinorEnum.GetItems,
            //        "Get items from {0} encountered an exception: {1}", new object[] { collection.Name, ex }, cursor.logger);
            //}
            finally
            {
                LoggerWrapper(MajorEnum.Information, MinorEnum.GetItems,
                    "Get {0} items from {1}", new object[] { items.Count, collection.Name }, cursor.Logger);
            }
        }

        public override void PurgeItems(IEnumerable<ExternalIDType> externalIDs, CollectionType collection, GetSerializerDelegate getSerializer)
        {
            ISerializerType serializer = getSerializer (SerializerKind);

            if (!LoggerWrapper(MajorEnum.Trace, MinorEnum.ActionStart,
                "Started {0}", new object[] { MinorEnum.PurgeItems }, serializer.Logger))
                return;

            MongoCollection  mongoCollection = _mongoDB[collection.Name];

            int deletedCount = 0;
            int deleteAttempts = 0;
            foreach (ExternalIDType externalID in externalIDs)
            {
                int retryCount = 0;
                Exception lastException = null;
                while (retryCount < 5)
                {
                    try
                    {
                        SafeModeResult result = mongoCollection.Remove(new QueryDocument("_id", externalID.ToString()),RemoveFlags.Single, SafeMode.True);

                        RemoveItemsStreamResolver(serializer, collection, externalID.ToString());

                        LoggerWrapper(MajorEnum.Trace, MinorEnum.ItemDeleted, "deleted item {0}",
                                      new object[] {externalID},
                                      serializer.Logger);

                        deletedCount += result.DocumentsAffected;

                        deleteAttempts++;

                        break;
                    }
                    catch (MongoConnectionException ex)
                    {
                        retryCount++;
                        lastException = ex;
                        LoggerWrapper(MajorEnum.Warning, MinorEnum.ErrorPurgingItems, "Error purging item {0} retrying for the {1} time, exception was: ", new object[] { externalID, retryCount, ex }, serializer.Logger);
                        Thread.Sleep((int)Math.Pow(10, retryCount));
                    }
                }
                if (retryCount == 5)
                {
                    LoggerWrapper(MajorEnum.Error, MinorEnum.RetryCountExceeded, "retry count exceeded deleting {0}: {1}", new object[] { externalID, lastException }, serializer.Logger);
                }
            }
            if (Cache != null)
                Cache.PurgeItems(externalIDs, collection, getSerializer);

            if (deletedCount == 0)
                LoggerWrapper(MajorEnum.Warning, MinorEnum.NothingDeleted, "Nothing deleted from {0}", new object[] { collection.Name }, serializer.Logger);

            LoggerWrapper(MajorEnum.Trace, MinorEnum.ItemsActuallyPurged,
                "{0} items actually deleted from {1}", new object[] { deletedCount, collection.Name }, serializer.Logger);
            LoggerWrapper(MajorEnum.Information, MinorEnum.PurgeItems,
                "{0} items deleted from {1}", new object[] { deleteAttempts, collection.Name }, serializer.Logger);
        }

        public override void PurgeItems(IBooleanValue selectionExpression, CollectionType collection, CursorType cursor)
        {
            if (!LoggerWrapper(MajorEnum.Trace, MinorEnum.ActionStart,
                "Started {0}", new object[] { MinorEnum.GetItems }, cursor.Logger))
                return;

            try
            {
                ISerializerType serializer = cursor.GetSerializer(SerializerKindEnum.CIP4Json);

                string lookForExternalId = GetExternalIdValueFromSelectionExpression(selectionExpression, serializer);

                if (Cache != null)
                {
                    Cache.PurgeItems(selectionExpression, collection, cursor);
                }


                if (!string.IsNullOrEmpty(lookForExternalId))
                {
                    try
                    {
                        MongoCollection mongoCollection = _mongoDB[collection.Name];

                        QueryDocument qdoc = new QueryDocument(new BsonElement("_id", lookForExternalId));

                        int retryCount = 0;

                        Exception originalException = null;

                        do
                        {
                            try
                            {
                                FindAndModifyResult findResult = mongoCollection.FindAndRemove(qdoc, null);
                                // returns the deleted document - maybe faster just to do a remove
                                cursor.RecordCount = findResult.Ok ? 1 : 0;

                                break;
                            }
                            catch (MongoSafeModeException mex)
                            {
                                LoggerWrapper(MajorEnum.Error, MinorEnum.ItemsUpdated, "Error deleting item {0}: {1}",
                                              new object[] { lookForExternalId, mex.Message }, serializer.Logger);
                                break;

                            }
                            catch (Exception ex)
                            {
                                if (retryCount++ > 4)
                                    throw new ApplicationException(
                                        "retrycount exceeded - innerexception is the original exception", originalException);

                                LoggerWrapper(MajorEnum.Warning, MinorEnum.ErrorPurgingItems, "Error purging item {0} retrying for the {1} time, exception was: ", new object[] { lookForExternalId, retryCount, ex }, serializer.Logger);

                                if (originalException == null)
                                    originalException = ex;

                                Thread.Sleep(10 ^ retryCount);
                            }

                        } while (true); 
                        return;
                    }
                    catch (FileNotFoundException)
                    {
                        LoggerWrapper(MajorEnum.Warning, MinorEnum.NothingDeleted, "Purge query didn't find anything {0}", new object[] { selectionExpression }, cursor.Logger);
                        return;
                    }
                }

                {
                    LoggerResponseEnum whatToDo = LoggerResponseEnum.keepGoing;
                    QueryDocument jsonQuery = BuildQuery(
                        selectionExpression,
                        collection,
                        serializer,
                        delegate(MajorEnum status, MinorEnum minorStatus, string message, object[] paras)
                        {
                            cursor.Logger(status, minorStatus, message, paras);
                            if (status == MajorEnum.Error)
                                whatToDo = LoggerResponseEnum.giveUp;
                            return LoggerResponseEnum.keepGoing;
                        });

                    cursor.Logger(MajorEnum.Trace, MinorEnum.QueryText, "Mongo query:{0}",
                                  new object[] { jsonQuery.ToJson() });

                    if (whatToDo == LoggerResponseEnum.giveUp)
                        return;

                    if (cursor.FailOnLinearSearch && IsLinearSearch(jsonQuery, serializer))
                        if (!LoggerWrapper(MajorEnum.Error, MinorEnum.FailOnLinearSearch,
                            "Failed on linear search accessing {0}", new object[] { collection }, cursor.Logger))
                            return;

                        int retryCount = 0;

                        Exception originalException = null;

                        do
                        {
                            try
                            {
                                MongoCollection mongoCollection = _mongoDB[collection.Name];

                                SafeModeResult safeResult = mongoCollection.Remove(jsonQuery, SafeMode.True);

                                cursor.RecordCount = safeResult.DocumentsAffected;

                                break;
                            }
                            catch (MongoSafeModeException mex)
                            {
                                LoggerWrapper(MajorEnum.Error, MinorEnum.ErrorPurgingItems, "Error deleting {0}: {1}",
                                              new object[] { selectionExpression, mex.Message }, serializer.Logger);
                                break;

                            }
                            catch (Exception ex)
                            {
                                if (retryCount++ > 4)
                                    throw new ApplicationException(
                                        "retrycount exceeded - innerexception is the original exception", originalException);

                                LoggerWrapper(MajorEnum.Warning, MinorEnum.ErrorPurgingItems, "Error purging items {0} retrying for the {1} time, exception was: ", new object[] { selectionExpression, retryCount, ex }, serializer.Logger);

                                if (originalException == null)
                                    originalException = ex;

                                Thread.Sleep(10 ^ retryCount);
                            }

                        } while (true); 

                    return;
                }
            }
            finally
            {
                LoggerWrapper(MajorEnum.Trace, MinorEnum.ItemsActuallyPurged,
                    "{0} items deleted from {1}", new object[] { cursor.RecordCount, collection.Name }, cursor.Logger);
                LoggerWrapper(MajorEnum.Information, MinorEnum.PurgeItems,
                    "{0} items deleted from {1}", new object[] { cursor.RecordCount, collection.Name }, cursor.Logger);
            }
        }

        public void SaveItems(IEnumerable<object> items, CollectionType collection, GetSerializerDelegate getSerializer)
        {
            ISerializerType serializer = getSerializer (SerializerKind);

            if (!LoggerWrapper(MajorEnum.Trace, MinorEnum.ActionStart,
                "Started {0}", new object[] { MinorEnum.SaveItems }, serializer.Logger))
                return;

            MongoCollection  mongoCollection = _mongoDB[collection.Name];

            int itemCount = 0;

            List<BsonDocument> insertThese = new List<BsonDocument>();

            foreach (object itemObject in items)
            {
                long versionNumber = serializer.GetVersionNumber(itemObject);

                // if the object already exists (implied by the VersionNumber being > 0) or
                // it has a populated stream property, then save it on its own
                if (serializer.StreamSeeds.Count > 0 || versionNumber == -1)
                {
                    itemCount += SaveNewNoVersion(serializer, collection, itemObject, mongoCollection);
                }
                else if (serializer.StreamSeeds.Count > 0 || versionNumber > 0)
                {
                    itemCount += SaveWithStreams(serializer, collection, itemObject, versionNumber, mongoCollection);
                }
                else
                {
                    BsonDocument itemDoc = (BsonDocument)(serializer).GetDataForObject(itemObject);

                    insertThese.Add(itemDoc);

                    if (insertThese.Count > 100)
                    {
                        itemCount += InsertBatch(serializer, mongoCollection, insertThese);
                        insertThese.Clear();
                    }
                }
            }

            if (insertThese.Count > 0)
                itemCount += InsertBatch(serializer, mongoCollection, insertThese);

            if (Cache != null)
                Cache.SaveItems(items, collection, getSerializer);

            LoggerWrapper(MajorEnum.Information, MinorEnum.SaveItems,
                "{0} items saved in {1}", new object[] { itemCount, collection.Name }, serializer.Logger);
        }

        private int SaveWithStreams(ISerializerType serializer, CollectionType collection, object itemObject, long versionNumber, MongoCollection mongoCollection)
        {
            int retryCount = 0;

            Exception originalException = null;

            int itemCount = 0;

            do
            {
                try
                {
                    BsonDocument itemDoc = (BsonDocument) (serializer).GetDataForObject(itemObject);

                    SaveItemsStreamResolver(serializer, collection, itemDoc["_id"].ToString());

                    QueryDocument query = new QueryDocument();

                    query.Add("_id", BsonValue.Create(serializer.GetExternalID(itemObject).ToString()));

                    if (!string.IsNullOrEmpty(serializer.GetVersionPropertyPath()) && versionNumber > 0)
                        query.Add(serializer.GetVersionPropertyPath(),versionNumber);

                    UpdateDocument update = new UpdateDocument(itemDoc);

                    SafeModeResult saveResult = mongoCollection.Update(query, update, SafeMode.True);

                    if (saveResult.DocumentsAffected == 0)
                    {
                        serializer.Logger(MajorEnum.Error, MinorEnum.StaleItemOnSave,
                            "The Item {0} has been updated by another user",
                            new object[] {serializer.GetExternalID(itemObject).ToString()});
                    }
                    else
                    {
                        itemCount++;
                        LoggerWrapper(MajorEnum.Trace, MinorEnum.ItemsUpdated, "Item {0} updated",
                            new object[] {itemDoc["_id"]}, serializer.Logger);
                    }
                    break;
                }
                catch (MongoSafeModeException mex)
                {
                    LoggerWrapper(MajorEnum.Error, MinorEnum.ItemsUpdated, "Error inserting item {0}",
                                  new object[] { mex.Message }, serializer.Logger);
                    break;

                }
                catch (Exception ex)
                {
                    if (retryCount++ > 4)
                        throw new ApplicationException(
                            "retrycount exceeded - innerexception is the original exception", originalException);

                    if (originalException == null)
                        originalException = ex;

                    Thread.Sleep(10 ^ retryCount);
                }
            } while (true); 
        

            return itemCount;
        }

        private int SaveNewNoVersion(ISerializerType serializer, CollectionType collection, object itemObject, MongoCollection mongoCollection)
        {
            int itemCount = 0;

            int retryCount = 0;

            Exception originalException = null;

            do
            {
                try
                {
                    BsonDocument itemDoc = (BsonDocument)(serializer).GetDataForObject(itemObject);

                    SaveItemsStreamResolver(serializer, collection, itemDoc["_id"].ToString());

                    SafeModeResult saveResult = mongoCollection.Save(itemDoc, SafeMode.True);

                    if (saveResult.DocumentsAffected == 0)
                        LoggerWrapper(MajorEnum.Error, MinorEnum.SaveDidNotHappen, "Item {0} was not saved",
                                        new object[] { itemDoc["_id"] }, serializer.Logger);


                    itemCount++;

                    LoggerWrapper(MajorEnum.Trace, MinorEnum.ItemsUpdated, "Item {0} saved",
                                    new object[] {itemDoc["_id"]}, serializer.Logger);
                    break;
                }
                catch (MongoSafeModeException mex)
                {
                    LoggerWrapper(MajorEnum.Error, MinorEnum.ItemsUpdated, "Error inserting item {0}",
                                  new object[] { mex.Message }, serializer.Logger);
                    break;

                }
                catch (Exception ex)
                {
                    if (retryCount++ > 4)
                        throw new ApplicationException("retrycount exceeded - innerexception is the original exception", originalException);

                    if (originalException == null)
                        originalException = ex;

                    Thread.Sleep(10 ^ retryCount);
                }

            } while (true);

            return itemCount;
        }

        private int InsertBatch(ISerializerType serializer, MongoCollection mongoCollection, List<BsonDocument> insertThese)
        {
            int retryCount = 0;

            int itemCount = 0; 
            
            Exception originalException = null;

            do
            {
                try
                {
#if FEELINGCAUTIOUS
                    IEnumerable<SafeModeResult> saveResults = mongoCollection.InsertBatch(insertThese, SafeMode.True);

                    foreach (SafeModeResult saveResult in saveResults)
                    {
                        if (saveResult.HasLastErrorMessage)
                            LoggerWrapper(MajorEnum.Error, MinorEnum.ItemsUpdated, "Error inserting item {0}",
                                          new object[] {saveResult.ToBsonDocument().ToJson()}, serializer.Logger);
                    }
#else
                    mongoCollection.InsertBatch(insertThese, SafeMode.False);
#endif
                    itemCount = insertThese.Count;
                    break;
                }
                catch(MongoSafeModeException mex)
                {
                    LoggerWrapper(MajorEnum.Error, MinorEnum.ItemsUpdated, "Error inserting item {0}",
                                  new object[] { mex.Message }, serializer.Logger);
                    break;

                }
                catch (Exception ex)
                {
                    if (retryCount++ > 4)
                        throw new ApplicationException("retrycount exceeded - innerexception is the original exception", originalException);

                    if (originalException == null)
                        originalException = ex;

                    Thread.Sleep(10 ^ retryCount);
                }

            } while (true);
            return itemCount;
        }

        protected static byte[] DecodeHex(string val)
        {
            var numberChars = val.Length;

            var bytes = new byte[numberChars / 2];
            for (var i = 0; i < numberChars; i += 2)
                try
                {
                    bytes[i / 2] = Convert.ToByte(val.Substring(i, 2), 16);
                }
                catch
                {
                    //failed to convert these 2 chars, they may contain illegal charracters
                    bytes[i / 2] = 0;
                }
            return bytes;
        }

        public void Terminate(LoggerDelegate logger)
        {
        }

        #endregion

        #region Helper Functions

        private void RemoveItemsStreamResolver(ISerializerType serializer, CollectionType collection, string externalID)
        {
            foreach (KeyValuePair<string, KeyValuePair<PropertyInfo, object>> streamSeed in serializer.StreamSeeds)
            {
                string qualifiedName = externalID + "." + streamSeed.Key;
                MongoGridFS fs = 
                    new MongoGridFS(
                        _mongoDB, 
                        new MongoGridFSSettings { Root = collection.Name + "_filedata"});
                MongoGridFSFileInfo gfi = new MongoGridFSFileInfo(fs, qualifiedName);
                gfi.Delete();
            }
        }

        private void GetItemsStreamResolver(ISerializerType serializer, CollectionType collection, string externalID)
        {
            foreach (KeyValuePair<string, KeyValuePair<PropertyInfo, object>> streamSeed in serializer.StreamSeeds)
            {
                try
                {
                    if (streamSeed.Value.Value == null)
                        continue;
                    string qualifiedName = externalID + "." + streamSeed.Key;
                    MongoGridFS fs = 
                        new MongoGridFS(
                            _mongoDB,
                            new MongoGridFSSettings { Root = collection.Name + "_filedata" } );
                    MongoGridFSStream gfs = fs.OpenRead(Path.GetFileName(qualifiedName));
                    streamSeed.Value.Key.SetValue(streamSeed.Value.Value, gfs, new object[] { });
                }
                catch (DirectoryNotFoundException ex)
                {
                    serializer.Logger(MajorEnum.Warning, MinorEnum.Generic, "Cannot find file for property, file path is {0} - {1}", new object[] { externalID + "." + streamSeed.Key, ex.Message });
                }
            }
        }

        private void SaveItemsStreamResolver(ISerializerType serializer, CollectionType collection, string externalID)
        {
            foreach (KeyValuePair<string, KeyValuePair<PropertyInfo, object>> streamSeed in serializer.StreamSeeds)
            {
                if (streamSeed.Value.Value == null)
                    continue;
                string qualifiedName = externalID + "." + streamSeed.Key;
                MongoGridFS fs = 
                    new MongoGridFS(
                        _mongoDB,
                        new MongoGridFSSettings {Root = collection.Name + "_filedata"});
                using (MongoGridFSStream gfs = fs.Create(Path.GetFileName(qualifiedName)))
                {
                    Stream source = (Stream) streamSeed.Value.Value;
                    byte[] buffer = new byte[2048]; // pick size
                    int bytesRead;
                    while ((bytesRead = source.Read(buffer, 0, buffer.Length)) > 0)
                    {
                        gfs.Write(buffer, 0, bytesRead);
                    }
                    source.Close();
                    gfs.Close();
                    source.Close();
                    source.Dispose();
                }
            }
        }

        private static bool IsLinearSearch(BsonDocument jsonQuery, ISerializerType serializer)
        {
            int keyCount = 0;
            foreach (string name in jsonQuery.Names)
            {
                if (name == "$or")
                    return true;
                if (name == "$not")
                    return true;
                string name1 = name;
                if (serializer.GetKeys().Any(k => name1.EndsWith("." + k.Key)))
                    keyCount++;
            }
            return keyCount == 0;
        }

        private QueryDocument BuildQuery(IBooleanValue selectionExpression, CollectionType collection, ISerializerType serializer, LoggerDelegate logger)
        {
            if (selectionExpression == null)
                return new QueryDocument();
            List<object> linearizedExpression = new List<object>();

            if (selectionExpression is LogicalExpressionType)
            {
                LinearizeExpression(
                    ((LogicalExpressionType)selectionExpression).ExpressionOperator,
                    (IBooleanValue)((LogicalExpressionType)selectionExpression).Operands[0],
                    (IBooleanValue)(((LogicalExpressionType)selectionExpression).Operands.Count > 1 ? ((LogicalExpressionType)selectionExpression).Operands[1] : null),
                    linearizedExpression, serializer.Logger);
                linearizedExpression.Add(((LogicalExpressionType)selectionExpression).ExpressionOperator);
            }
            else
                linearizedExpression.Add(selectionExpression); // assume its a BooleanEXpressionType


            BsonArray clauses = new BsonArray();
            QueryDocument jsonQuery = new QueryDocument();
            foreach (object exprElement in linearizedExpression)
            {
                if (exprElement is LogicalOperatorsEnum)
                {
                    if ((LogicalOperatorsEnum)exprElement == LogicalOperatorsEnum.OR)
                        jsonQuery.Add("$or", clauses);
                    if ((LogicalOperatorsEnum)exprElement == LogicalOperatorsEnum.NOT)
                        jsonQuery.Add("$not", clauses);
                    if ((LogicalOperatorsEnum)exprElement == LogicalOperatorsEnum.AND)
                        foreach (BsonDocument doc in clauses)
                            foreach (string name in doc.Names)
                                jsonQuery.Add(name, doc[name]);
                }
                else if (exprElement is BooleanExpressionType)
                    clauses.Add(BuildQuery(new BsonDocument(), (BooleanExpressionType)exprElement, collection, serializer, logger));
                else if (exprElement is BooleanAtomType)
                    clauses.Add(BuildQuery(new BsonDocument(), (BooleanAtomType)exprElement, collection, serializer, logger));
                else
                    LoggerWrapper(MajorEnum.Error, MinorEnum.InvalidQuery, "Logical expression parameters must be a logical expression, a boolean expression or a boolean atom - scanning: {0}", new object[] { exprElement.ToString() }, logger);
            }
            if (jsonQuery.Names.Count() == 0)
            {
                foreach (BsonDocument doc in clauses)
                    foreach (string name in doc.Names)
                        jsonQuery.Add(name, doc[name]);
            }
            return jsonQuery;
        }

        private static void LinearizeExpression(LogicalOperatorsEnum op, IBooleanValue lExpression, IBooleanValue rExpression, List<object> expression, LoggerDelegate logger)
        {
            if (op != LogicalOperatorsEnum.NOT)
            {
                if (lExpression is LogicalExpressionType)
                {
                    if (((LogicalExpressionType)lExpression).ExpressionOperator != op)
                    {
                        logger(MajorEnum.Error, MinorEnum.Generic, "Cannot linearize {0} {1} {2}", new object[] { rExpression.ToString(), op.ToString(), lExpression.ToString() });
                        return;
                    }
                    LinearizeExpression(
                        ((LogicalExpressionType)lExpression).ExpressionOperator,
                        (IBooleanValue)((LogicalExpressionType)lExpression).Operands[0],
                        (IBooleanValue)((LogicalExpressionType)lExpression).Operands[1],
                        expression, logger);
                }
                else // assume its a BooleanExpressionType
                {
                    expression.Add(lExpression);
                }
            }
            else if (op == LogicalOperatorsEnum.NOT)
            {
                expression.Add(((LogicalExpressionType)lExpression).ExpressionOperator);
                LinearizeExpression(
                    ((LogicalExpressionType)lExpression).ExpressionOperator,
                    (IBooleanValue)((LogicalExpressionType)lExpression).Operands[0],
                    (IBooleanValue)((LogicalExpressionType)lExpression).Operands[1],
                    expression, logger);
                expression.Add(op);
                return;
            }
            if (rExpression is LogicalExpressionType)
            {
                if (((LogicalExpressionType)rExpression).ExpressionOperator != op)
                {
                    logger(MajorEnum.Error, MinorEnum.Generic, "Cannot linearize {0} {1} {2}", new object[] { rExpression.ToString(), op.ToString(), lExpression.ToString() });
                }
                LinearizeExpression(
                    ((LogicalExpressionType)rExpression).ExpressionOperator,
                    (IBooleanValue)((LogicalExpressionType)rExpression).Operands[0],
                    (IBooleanValue)((LogicalExpressionType)rExpression).Operands[1],
                    expression, logger);
            }
            else // assume its a BooleanExpressionType
            {
                expression.Add(rExpression);
            }
            return;
        }

/*        private BsonDocument BuildQuery(BsonDocument jsonQuery, RulesEngine.LogicalExpressionType selectionExpression, CollectionType collection, LoggerDelegate logger)
        {
            if (selectionExpression.ExpressionOperator == LogicalOperatorsEnum.AND)
            {
                foreach (int index in new[] { 0, 1 })
                {
                    if (selectionExpression.Operands[index] is LogicalExpressionType)
                        BuildQuery(jsonQuery, (LogicalExpressionType)selectionExpression.Operands[index], collection, logger);
                    else if (selectionExpression.Operands[index] is BooleanExpressionType)
                        BuildQuery(jsonQuery, (BooleanExpressionType)selectionExpression.Operands[index], collection, logger);
                    else if (selectionExpression.Operands[index] is BooleanAtomType)
                        BuildQuery(jsonQuery, (BooleanAtomType)selectionExpression.Operands[index], collection, logger);
                    else
                        LoggerWrapper(MajorEnum.Error, MinorEnum.InvalidQuery, "Logical expression parameters must be a logical expression, a boolean expression or a boolean atom - scanning: {0}", new object[] { selectionExpression.Operands[index].ToString() }, logger);
                }
            }
            else if (selectionExpression.ExpressionOperator == LogicalOperatorsEnum.OR)
            {
                BsonArray orDocs = new BsonArray();
                foreach (int index in new[] { 0, 1 })
                {
                    BsonDocument orDoc = new BsonDocument();
                    if (selectionExpression.Operands[index] is LogicalExpressionType)
                        BuildQuery(orDoc, (LogicalExpressionType)selectionExpression.Operands[index], collection, logger);
                    else if (selectionExpression.Operands[index] is BooleanExpressionType)
                        BuildQuery(orDoc, (BooleanExpressionType)selectionExpression.Operands[index], collection, logger);
                    else if (selectionExpression.Operands[index] is BooleanAtomType)
                        BuildQuery(orDoc, (BooleanAtomType)selectionExpression.Operands[index], collection, logger);
                    else
                        LoggerWrapper(MajorEnum.Error, MinorEnum.InvalidQuery, "Logical expression parameters must be a logical expression, a boolean expression or a boolean atom - scanning: {0}", new object[] { selectionExpression.Operands[index].ToString() }, logger);
                    orDocs.Add(orDoc);
                }
                jsonQuery.Add("$or", orDocs);
            }
            else if (selectionExpression.ExpressionOperator == LogicalOperatorsEnum.NOT)
            {
                BsonDocument notDoc = new BsonDocument();
                if (selectionExpression.Operands[0] is LogicalExpressionType)
                    BuildQuery(jsonQuery, (LogicalExpressionType)selectionExpression.Operands[0], collection, logger);
                else if (selectionExpression.Operands[0] is BooleanExpressionType)
                    BuildQuery(jsonQuery, (BooleanExpressionType)selectionExpression.Operands[0], collection, logger);
                else if (selectionExpression.Operands[0] is BooleanAtomType)
                    BuildQuery(jsonQuery, (BooleanAtomType)selectionExpression.Operands[0], collection, logger);
                else
                    LoggerWrapper(MajorEnum.Error, MinorEnum.InvalidQuery, "Logical expression parameters must be a logical expression, a boolean expression or a boolean atom - scanning: {0}", new object[] { selectionExpression.Operands[0].ToString() }, logger);
                jsonQuery.Add("$not", notDoc);
            }
            return jsonQuery;
        }
*/

        private static object GetLiteralValue(AtomType atom)
        {
            // ReSharper disable PossibleInvalidOperationException
            if (!atom.IsLiteral)
                throw new ArgumentException(string.Format("The atom {0} is of an unexpected type - literal expected", atom));
            if (atom is DateTimeAtomType)
                return DateTime.SpecifyKind(((DateTimeAtomType)atom).AtomValue.Value, DateTimeKind.Local);
            if (atom is StringAtomType)
                return ((StringAtomType)atom).AtomValue;
            if (atom is IntegerAtomType)
                return ((IntegerAtomType)atom).AtomValue.Value;
            if (atom is BooleanAtomType)
                return ((BooleanAtomType)atom).AtomValue.Value;
            if (atom is FloatAtomType)
                return ((FloatAtomType)atom).AtomValue.Value;
            return atom.ToString();
            // ReSharper restore PossibleInvalidOperationException
        }
        
        private static BooleanExpressionType TransformSOME(BooleanExpressionType selectionExpression)
        {
            // if its a SOME, rework the quantifier expression so its named values use the 
            // SOME's SetAtom name and return that - MongoDb query works in such a way that 
            // it acts like SOME where an expression involes scalar EQL vector 
            if (selectionExpression.ExpressionOperator == BooleanOperatorsEnum.SOME)
            {
                SetAtomType set = (SetAtomType)selectionExpression.Operands[0];

                BooleanExpressionType subExpr = (BooleanExpressionType)selectionExpression.Operands[1];

                BooleanExpressionType newBool = new BooleanExpressionType();

                newBool.ExpressionOperator = subExpr.ExpressionOperator;

                // The expression gets messed up so's MongoDB will understand it
                // The CompareProperties function needs the original form so do
                // Clone in what follows rather than just taking references
                newBool.Operands.Add(subExpr.Operands[0].Clone());

                newBool.Operands.Add(subExpr.Operands[1].Clone());

                if (subExpr.Operands[0].IsNamedValue)
                    newBool.Operands[0].ResolveName(set.AtomName);

                if (subExpr.Operands[1].IsNamedValue)
                    newBool.Operands[1].ResolveName(set.AtomName);

                return newBool;

            }

            return selectionExpression;
        }

        private BsonDocument BuildQuery(BsonDocument jsonQuery, BooleanExpressionType selectionExpressionIn, CollectionType collection, ISerializerType serializer, LoggerDelegate logger)
        {
            if (selectionExpressionIn.ExpressionOperator == BooleanOperatorsEnum.ALL)
                return jsonQuery;  // todo: could use $all

            BooleanExpressionType selectionExpression = TransformSOME(selectionExpressionIn);

            AtomType namedValue =
                selectionExpression.Operands[0].IsNamedValue ? (AtomType)selectionExpression.Operands[0] :
                selectionExpression.Operands[1].IsNamedValue ? (AtomType)selectionExpression.Operands[1] : null;

            AtomType literalValue =
                selectionExpression.Operands[0].IsLiteral ? (AtomType)selectionExpression.Operands[0] :
                selectionExpression.Operands[1].IsLiteral ? (AtomType)selectionExpression.Operands[1] : null;

            if (namedValue == null || literalValue == null)
            {
                LoggerWrapper(MajorEnum.Error, MinorEnum.InvalidQuery, "Invalid boolean expression. Each boolean clause must consist of a literal and a named value: {0}", new object[] { selectionExpression.ToString() }, logger);
                return jsonQuery;
            }

            string path = JsonTypedPath(namedValue.AtomName, collection);

            path = serializer.GetSerializeAsName(path);

            switch (selectionExpression.ExpressionOperator)
            {
                case BooleanOperatorsEnum.EQL:
                    jsonQuery.Add(path, BsonValue.Create(GetLiteralValue(literalValue)));
                    break;
                case BooleanOperatorsEnum.NEQ:
                    jsonQuery.Add(path, new BsonDocument("$ne", BsonValue.Create(GetLiteralValue(literalValue))));
                    break;
                case BooleanOperatorsEnum.GEQ:
                    if (namedValue == selectionExpression.Operands[0])
                        jsonQuery.Add(path, new BsonDocument("$gte", BsonValue.Create(GetLiteralValue(literalValue))));
                    else
                        jsonQuery.Add(path, new BsonDocument("$lte", BsonValue.Create(GetLiteralValue(literalValue))));
                    break;
                case BooleanOperatorsEnum.GTR:
                    if (namedValue == selectionExpression.Operands[0])
                        jsonQuery.Add(path, new BsonDocument("$gt", BsonValue.Create(GetLiteralValue(literalValue))));
                    else
                        jsonQuery.Add(path, new BsonDocument("$lt", BsonValue.Create(GetLiteralValue(literalValue))));
                    break;
                case BooleanOperatorsEnum.LEQ:
                    if (namedValue == selectionExpression.Operands[0])
                        jsonQuery.Add(path, new BsonDocument("$lte", BsonValue.Create(GetLiteralValue(literalValue))));
                    else
                        jsonQuery.Add(path, new BsonDocument("$gte", BsonValue.Create(GetLiteralValue(literalValue))));
                    break;
                case BooleanOperatorsEnum.LSS:
                    if (namedValue == selectionExpression.Operands[0])
                        jsonQuery.Add(path, new BsonDocument("$lt", BsonValue.Create(GetLiteralValue(literalValue))));
                    else
                        jsonQuery.Add(path, new BsonDocument("$gt", BsonValue.Create(GetLiteralValue(literalValue))));
                    break;
                case BooleanOperatorsEnum.STARTSWITH:
                    if (namedValue != selectionExpression.Operands[0])
                    {
                        LoggerWrapper(MajorEnum.Error, MinorEnum.InvalidQuery, "Invalid boolean expression. startswith operator must have a named value as it's first parameter: {0}", new object[] { selectionExpression.ToString() }, logger);
                        return jsonQuery;
                    }
                    BsonDocument startsWithQueryDoc = new BsonDocument("$regex", Regex.Escape(string.Format("^{0}", GetLiteralValue(literalValue))));
                    jsonQuery.Add(path, startsWithQueryDoc);
                    break;
                case BooleanOperatorsEnum.CONTAINS:
                    if (namedValue != selectionExpression.Operands[0])
                    {
                        LoggerWrapper(MajorEnum.Error, MinorEnum.InvalidQuery, "Invalid boolean expression. Contains operator must have a named value as it's first parameter: {0}", new object[] { selectionExpression.ToString() }, logger);
                        return jsonQuery;
                    }
                    BsonDocument containsQueryDoc = new BsonDocument("$regex", Regex.Escape(GetLiteralValue(literalValue).ToString()));
                    jsonQuery.Add(path, containsQueryDoc);
                    break;
            }
            return jsonQuery;
        }

        private BsonDocument BuildQuery(BsonDocument jsonQuery, BooleanAtomType selectionExpression, CollectionType collection, ISerializerType serializer, LoggerDelegate logger)
        {
            if (selectionExpression.AtomSource == AtomSourceEnum.namedValue)
            {
                string path = JsonTypedPath(selectionExpression.AtomName, collection);
                jsonQuery.Add(serializer.GetSerializeAsName(path), "true");
            }
            else if (selectionExpression.AtomSource == AtomSourceEnum.literal)
            {
                if (selectionExpression.AtomValue.HasValue && !selectionExpression.AtomValue.Value)
                    jsonQuery.Add("false", "true");
            }
            else
                LoggerWrapper(MajorEnum.Error, MinorEnum.InvalidQuery, "Invalid clause in the selection expression- the clause {0} is not of an expected type - only literals or named values are allowed", new object[] { selectionExpression }, logger);
            return jsonQuery;
        }

        private static string JsonTypedPath(string path, CollectionType collection)
        {
            if (path.ToLower() == "externalid")
                return "_id";
            return collection.CollectionCompoundType.Name + JsonTypedPath(new List<string>(path.Split('.')), collection.CollectionCompoundType);
        }

        private static string JsonTypedPath(List<string> path, Type aType)
        {
            path.RemoveAt(0);
            if (path.Count == 0)
                return string.Empty;
            PropertyInfo prop = GetActualType(aType).GetProperty(path[0]);
            string suffix = ValueType(prop.PropertyType) ? string.Empty : ("." + GetActualType(prop.PropertyType).Name);
            return "." + path[0] + suffix + JsonTypedPath(path, prop.PropertyType);
        }

        private static bool ValueType(Type type)
        {
            Type actualType = GetActualType(type);
            if (actualType.IsEnum)
                return true;
            if (actualType.Name == "String")
                return true;
            return type.IsValueType;
        }

        private static Type GetActualType(Type t)
        {
            Type actualType = Nullable.GetUnderlyingType(t);
            actualType = actualType ?? t;
            if (t.IsGenericType)
                actualType = t.GetGenericArguments()[0];
            if (t.IsArray)
                actualType = t.GetElementType();
            return actualType;
        }

        #endregion

    }
}
